package com.chance.cc.crawler.development.scripts.xiaohongshu.monitor;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.CrawlerRecordContext;
import com.chance.cc.crawler.core.downloader.*;
import com.chance.cc.crawler.core.pipeline.result.CrawlerData;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.core.record.CrawlerResultRecord;
import com.chance.cc.crawler.core.script.crawler.CrawlerCommonScript;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.time.DateFormatUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.RandomAccessFile;
import java.net.URLEncoder;
import java.util.*;

/**
 * @author bx
 * @date 2021/1/8 0008 12:11
 */
public class XHSSchedulerMonitorScript extends CrawlerCommonScript {

    private static final Logger logger = LoggerFactory.getLogger(XHSSchedulerMonitorScript.class);
    public static final String monitorUlr = "/scheduler/xhs-monitor";
    public static final String schedulerMonitorFilePathFormat = "/data/chance_crawler_test/logs/scheduler/data-node-scheduler-%s.all";
    public static final String signalFilePathFormat = "/data/chance_crawler_test/signal/node-scheduler-size-%s.txt";

    public static final String sendEmailUrl = "http://192.168.1.213:9030/api/chance/mail?type=xhs&content=%s";

    public static final Map<String,String> deviceSids = new HashMap<>();
    static {
        deviceSids.put("192.168.2.30:5555","session.1635129065545188465985");
        deviceSids.put("192.168.2.31:5555","session.1632993903415389261165");
        deviceSids.put("192.168.2.32:5555","session.1635150285862861030582");
        deviceSids.put("192.168.2.33:5555","session.1635732138835821040106");
        deviceSids.put("192.168.2.34:5555","session.1635148338860593314415");
        deviceSids.put("192.168.2.35:5555","session.1635733368323704040081");
        deviceSids.put("192.168.2.36:5555","session.1635935668480766152610");
    }

    private Set<String> lastSendContents = new HashSet<>();
    private long lastSendTime = 0;
    public static int downloadRetryCount = 4;
    public static int downloadRetryInterval = 1000;

    public List<CrawlerRequestRecord> parseLinks(CrawlerRecordContext context) {

        long currentTimeMillis = System.currentTimeMillis();
        String currentDate = DateFormatUtils.format(currentTimeMillis, "yyyy-MM-dd");
        String schedulerMonitorFilePath = String.format(schedulerMonitorFilePathFormat,currentDate);
        Downloader pageDownloader = context.getPageDownloader();

        try {
            Set<String> sendContents = new HashSet<>();

            String signalFilePath = String.format(signalFilePathFormat,currentDate);
            File signalFile = new File(signalFilePath);
            long lastTimeFileSize = 0;
            RandomAccessFile randomFile;
            if (!signalFile.exists()){
                //初始文件大小
                randomFile = new RandomAccessFile(schedulerMonitorFilePath,"rw");
                lastTimeFileSize = randomFile.length();
                randomFile.seek(lastTimeFileSize);
                IOUtils.write(String.valueOf(lastTimeFileSize),new FileOutputStream(signalFilePath),"utf-8");
                logger.info("scheduler log path {} init random file length {}",schedulerMonitorFilePath,lastTimeFileSize);
            } else {
                List<String> nodeLogSize = IOUtils.readLines(new FileInputStream(signalFilePath), "utf-8");
                if (nodeLogSize.size() > 0){
                    lastTimeFileSize = Long.valueOf(nodeLogSize.get(0));
                } else {
                    logger.error("scheduler log size file {} is error!",signalFilePath);
                }
                randomFile = new RandomAccessFile(schedulerMonitorFilePath,"rw");
                randomFile.seek(lastTimeFileSize);
                logger.info("scheduler log path {} last time file size {}",schedulerMonitorFilePath,lastTimeFileSize);
            }

            String line;
            while( (line = randomFile.readLine())!= null) {
                if (line.contains("XHSPhoneSearchForTimeLineMI5SADBScript") && line.contains("Jump out captcha")){
                    String[] itemArray = line.split(" - ");
                    sendContents.add(itemArray[itemArray.length-1]);
                }
            }
            lastTimeFileSize = randomFile.length();
            IOUtils.write(String.valueOf(lastTimeFileSize),new FileOutputStream(signalFilePath),"utf-8");
            logger.info("scheduler log path {} tail random file length {}",schedulerMonitorFilePath,lastTimeFileSize);
            HttpRequest httpRequest = new HttpRequest();
            if (sendContents.size() > 0){
                checkVerify(pageDownloader,sendContents); //校验验证码
            }

            if (sendContents.size() > 0){
                long endTime = System.currentTimeMillis();
                boolean isSend = false;
                if (lastSendContents.size() > 0){ //比较当前发送内容和上次发送内容差异，如果上次发送内容不包含本次发送内容，发送邮件
                    for (String sendContent : sendContents) {
                        if (!lastSendContents.contains(sendContent)){
                            isSend = true;
                            break;
                        }
                    }
                    if (!isSend){ // 如果没有新增的内容，判断具体上次发送时间是否超过1小时，超过则发送邮件
                        if ((endTime - lastSendTime) >3600000){ //1小时内只发送一次邮件
                            isSend = true;
                            lastSendTime = endTime;
                        }
                    }
                } else {
                    isSend = true;
                }
                lastSendContents.clear();
                lastSendContents.addAll(sendContents);

                if (isSend){
                    httpRequest.setUrl(String.format(sendEmailUrl, URLEncoder.encode(JSON.toJSONString(sendContents),"utf-8")));
                    HttpPage download = pageDownloader.download(httpRequest, HttpConfig.me(domain()));
                    logger.info("email sent result info {}",download.getRawText());
                }
            }

        } catch (Exception e) {
            logger.error(e.getMessage(),e);
        }
        return null;
    }

    private void checkVerify(Downloader pageDownloader,Set<String> sendContents){
        List<String> clearContents = new ArrayList<>();
        for (String sendContent : sendContents) {

            HttpRequest httpRequest = new HttpRequest();
            httpRequest.setUrl("http://192.168.1.212:9310/xhs/baidu/fverify");
            int retry = 1;
            while (retry <= downloadRetryCount){
                HttpPage httpPage = pageDownloader.download(httpRequest, HttpConfig.me(domain()));
                if (httpPage.isDownloadSuccess()){
                    try {
                        String riskLevel = httpPage.getJson().jsonPath("riskLevel").get();
                        logger.info("check verify download response:{}",httpPage.getRawText());
                        if ("PASS".equals(riskLevel)){//通过验证，提交rid
                            String rid = httpPage.getJson().jsonPath("rid").get();
                            if (submitRid(rid, pageDownloader, sendContent)){
                                clearContents.add(sendContent);
                                break;
                            }
                        }
                    } catch (Exception e) {
                        logger.error("Verification code verification failed: {}",e.getMessage());
                    }
                }

                try {
                    Thread.sleep(downloadRetryInterval);
                } catch (InterruptedException e) {
                    logger.error(e.getMessage());
                }

                retry++;
            }

            if(retry > downloadRetryCount){
                logger.error("check verify retry Limit exceeded!");
            }

        }

        if (clearContents.size() > 0){
            sendContents.removeAll(clearContents);
        }
    }

    private boolean submitRid(String rid,Downloader pageDownloader,String sendContent){
        String devices = sendContent.split("--")[1];
        String sid = deviceSids.get(devices);

        HttpRequest httpRequest = new HttpRequest();
        httpRequest.setUrl("https://www.xiaohongshu.com/api/sns/v1/system_service/slide_captcha_check");
        httpRequest.addHeader("Content-Type","application/x-www-form-urlencoded");
        httpRequest.addHeader("Host","www.xiaohongshu.com");
        httpRequest.setMethod("POST");

        Map<String,Object> bodyParam = new HashMap<>();
        bodyParam.put("pass","true");
        bodyParam.put("rid",rid);
        bodyParam.put("platform","android");
        bodyParam.put("sid",sid);
        logger.info("slide captcha check body param:{}",JSON.toJSONString(bodyParam));
        httpRequest.setRequestBody(HttpRequestBody.form(bodyParam, "utf-8"));
        HttpPage httpPage = pageDownloader.download(httpRequest, HttpConfig.me("android_submit_rid"));
        if (httpPage.isDownloadSuccess()){
            try {
                String passed = httpPage.getJson().jsonPath("data.passed").get();
                if ("true".equals(passed)){
                    return true;
                }
            } catch (Exception e) {
                logger.error("app submit rid fail,error:[{}]]",e.getMessage());
            }
        }
        logger.error("app submit rid error response:[{}]",httpPage.getRawText());
        return false;
    }

    @Override
    public List<CrawlerData> washPage(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        return null;
    }

    @Override
    public void initUrlRegulars() {
        addUrlRegular(monitorUlr);
    }

    @Override
    public boolean crawlerCheck(CrawlerRequestRecord crawlerRecord) {
        return true;
    }

    @Override
    public void afterExecute(CrawlerRecordContext context) {

    }

    @Override
    public String domain() {
        return "xhs-monitor";
    }

    /**
     *
     * 重写 crawlerProcess 以支持 download 下载
     * @param context
     */

    @Override
    public void crawlerProcess(CrawlerRecordContext context) {
        //parse request list
        if (context.getCrawlerRecord().isNeedParsedPage()){
            parsePage(context);
        }

        //need wash request
        if(context.getCrawlerRecord().isNeedWashPage()){
            washResult(context);
        }
    }

    private void parsePage(CrawlerRecordContext context){
        //parse page links
        List<CrawlerRequestRecord> links = parseLinks(context);
        //internal download
        internalDownload(context,links);
        //after internal download
        if (context.hasInternalDownloadLinks()){
            afterInternalDownload(context.getCrawlerRecord(),context.getInternalDownloadLinks(),links);
        }
        //add to context parsed links
        if (links != null && links.size() >0){
            for (CrawlerRequestRecord requestRecord : links){
                context.addCrawlerRecord(requestRecord);
            }
        }else{
            logger.warn("record [{}] parsed links is null!",context.getCrawlerRecord());
        }
    }

    @Override
    public List<CrawlerRequestRecord> parseLinks(CrawlerRequestRecord crawlerRecord, HttpPage page) {
        return null;
    }

    private void internalDownload(CrawlerRecordContext context,List<CrawlerRequestRecord> links){
        //extract internal download links
        List<CrawlerRequestRecord> internalDownloadLinks = new ArrayList<>();
        for (CrawlerRequestRecord requestRecord : links){
            if (requestRecord.tagsCreator().requestTags().hasRequestType(CrawlerEnum.CrawlerRequestType.internalDownload)){
                internalDownloadLinks.add(requestRecord);
            }
        }
        if (internalDownloadLinks.size()>0){
            links.removeAll(internalDownloadLinks);
        }
        //download links
        if (internalDownloadLinks == null || internalDownloadLinks.size()<=0){
            return;
        }
        for (CrawlerRequestRecord requestRecord : internalDownloadLinks){
            HttpPage page = context.getPageDownloader()
                    .download(requestRecord.getHttpRequest(),requestRecord.getHttpConfig());
            requestRecord.setInternalDownloadPage(page);
        }
        //set to context
        context.setInternalDownloadLinks(internalDownloadLinks);
    }

    private void washResult(CrawlerRecordContext context){
        CrawlerRequestRecord crawlerRecord = context.getCrawlerRecord();
        HttpPage page = context.getPage();

        List<CrawlerResultRecord> requestResults = new ArrayList<>();
        List<CrawlerData> crawlerDataList = washPage(crawlerRecord,page);
        if (crawlerDataList !=null && crawlerDataList.size()>0){
            for (CrawlerData crawlerData : crawlerDataList){
                //生成result record
                CrawlerResultRecord crawlerResultRecord = new CrawlerResultRecord();
                if (crawlerData.getFilter()!=null){
                    //filter info
                    crawlerResultRecord.setFilter(crawlerData.getFilter());
                    crawlerResultRecord.setFilterInfos(crawlerData.getFilterInfos());
                }
                crawlerResultRecord.setRecordKey(crawlerData.getDataId());
                crawlerResultRecord.setReleaseTime(crawlerData.getReleaseTime());
                //request tags
                crawlerResultRecord.setTags(crawlerData.getTags());
                //翻入过滤标签
                crawlerResultRecord.setFilterPipelineResult(crawlerData.isFilterPipelineResult());
                crawlerResultRecord.setJsonStr(JSON.toJSONString(crawlerData));
                requestResults.add(crawlerResultRecord);
            }
        }
        context.setRequestResults(requestResults);
    }
}
